import sys
import requests
import os
from lxml import etree
import re

def download(url,headers):
    for i in range(3):
        try:
            return requests.get(url, headers).content
        except Exception as e:
            print(e)

#####产品信息当前页下载####
def product_get(url,headers):

    ###获取目标xpath###
    page_text = requests.get(url=url, headers=headers).text
    tree = etree.HTML(page_text)
    type = tree.xpath('//div[@class="page-body"]/h1/text()')
    pic = tree.xpath('//div[@class="product-grid"]//a/img/@src')
    title = tree.xpath('//div[@class="product-grid"]//h2/a/text()')
    price = tree.xpath('//span[@class="price actual-price"]/text()')

    ###把所有信息合并到列表中######
    product = list(zip(title,price,pic))

    ####进入逐个逐个的安排#####
    for i in product:

        ###新建分类文件夹###
        fold = "./bobble/{}".format(type[0])
        if not os.path.exists(fold):
            os.mkdir(fold)

        #############按指定格式持久化#################
        ######图片命名#######
        img_name = i[0].replace(':', ' ').replace('"', 'inch') + "__" + i[1] + ".jpg"
        #####图片路径和命名汇总#######
        img_save = "{}/{}".format(fold, img_name)

        if os.path.exists(img_save):
            print(img_name, '已经存在！！')
            continue

        print(i[2], '开始下载！！')


        img_data = download(i[2], headers)
        if not img_data:
            continue

        ########保存下来######
        with open(img_save, 'wb') as fp:
            fp.write(img_data)
            print(img_name, '下载成功！！')


    ##############判断并爬取下一页##########套娃来了#################
    next_words = tree.xpath('//li[@class="next-page"]/a/@href')
    if len(next_words) > 0:
        next_url = "https://www.yesbobbleheads.com" + next_words[0]
        product_get(next_url, headers=headers)



#########获取首页分类
headers = {
    'user-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}

index_url = "https://www.yesbobbleheads.com/"

index = requests.get(url=index_url, headers=headers).text
tree = etree.HTML(index)
categories = ["https://www.yesbobbleheads.com"+i for i in tree.xpath('//div[@class="subnavi"]/a/@href')]


######开始爬取#####
for url in categories:
    product_get(url, headers)